import re
import json
import requests
import pymongo


class CsdnSpider(object):
    def __init__(self):
        self.headers = {'UserAgent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}

        # self.client = pymongo.MongoClient('localhost')
        # self.db = self.client['csdn2']

    def get_article_html(self):
        article_url = 'https://www.csdn.net/nav/career'
        response = requests.get(article_url, headers=self.headers)
        # print(response.text)
        shown_offset = re.findall(re.compile(r'<ul.*?id="feedlist_id" shown-offset="(.*?)">', re.S), response.text)[0]
        print(shown_offset)

        data = re.findall(re.compile(r'<li class="clearfix" data-type="blog" .*?<div class="title">.*?<a.*?>(.*?)</a>.*?<div class="summary oneline">(.*?)</div>', re.S), response.text)
        # print(data)
        for title, summary in data:
            new_title = title.replace('\n', '').replace(' ', '')
            new_summary = summary.replace('\n', '').replace(' ', '')

            new_dict = {"title": new_title, "summary": new_summary}
            print(new_dict)

        return shown_offset

    def get_json_data(self, shown_offset):
        api_url = 'https://www.csdn.net/api/articles?type=more&category=career&shown_offset={}'.format(shown_offset)
        print(api_url)
        json_dict = json.loads(requests.get(api_url, headers=self.headers).text)
        articles = json_dict.get('articles', None)

        for item in articles:
            next_title = item['title']
            next_summary = item['summary']
            dic = {"title": next_title, "summary": next_summary}
            print(dic)

        shown_offset = dic.get(shown_offset, None)
        if articles and len(articles) > 0 and shown_offset:
            self.get_json_data(shown_offset)


if __name__ == '__main__':
    cs = CsdnSpider()
    shownoffset = cs.get_article_html()
    cs.get_json_data(shownoffset)










